package com.kaigejava.webmaginc.csdn;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * @author kaigejava
 * CSDN下一页返回json数据处理
 */
public class CsdnNextPageJson implements PageProcessor {
    public void process(Page page) {
        System.out.println(page.getJson());
        String jsonStr = page.getJson().toString();
        JSONObject jsonObject =JSONObject.parseObject(jsonStr);
        boolean status = jsonObject.getBoolean("status");
        if(status){
            //获取数据集合
            JSONArray jsonArray =jsonObject.getJSONArray("articles");
            int size = jsonArray.size();
            for(int i = 0;i<size;i++){
             Object obj = jsonArray.get(i);
                JSONObject liInfo = jsonArray.getJSONObject(i);
                String url = liInfo.getString("url");
                System.out.println("url:"+url);
            }
        }
    }

    public Site getSite() {
        return Site.me().setSleepTime(100).setRetryTimes(3);
    }
    public static void main(String[] args) {
        Spider spider = Spider.create(new CsdnNextPageJson()).addUrl("https://blog.csdn.net/api/articles?type=more&category=java&shown_offset=0");
        spider.addRequest().run();
    }
}
